from bs4 import BeautifulSoup
import requests
import json
import arrow
import tushare as ts
from tqdm import tqdm
import time
import tempfile
import csv
import sys
sys.setrecursionlimit(10000) 
import logging
from functools import wraps
from collections import defaultdict
from config import get_header
import re

def get_log():
    log = logging.getLogger(__name__)
    ft = logging.Formatter('%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s','%Y-%m-%d %H:%M:%S')

    fh = logging.FileHandler('log/%s.log'%__name__)
    fh.setFormatter(ft)
    fh.setLevel(logging.WARNING)

    sh = logging.StreamHandler()
    sh.setFormatter(ft)
    sh.setLevel(logging.DEBUG)

    log.addHandler(fh)
    log.addHandler(sh)

    log.setLevel(logging.DEBUG)
    return log

log = get_log()

class setLog:
    @classmethod
    def request(cls, retry=3):
        def decorate(func):
            @wraps(func)
            def wrapper(*args, **kwargs):
                exception_count = defaultdict(lambda:0)
                for _ in range(retry):
                    try:
                        return func(*args, **kwargs)
                    except requests.exceptions.RequestException as e:
                        exception_count['RequestException'] += 1
                        log.warning(e)
                        time.sleep(5)
                    except AssertionError as e:
                        exception_count['AssertionError'] += 1
                        log.warning('%s %s'%(str(e),str(args if args else kwargs)))
                        if exception_count['AssertionError'] >= 1:
                            log.error('%s %s time over'%(str(e),str(args if args else kwargs)))
                            return
                        time.sleep(5)
                    except TypeError as e:
                        exception_count['TypeError'] += 1
                        log.warning('%s %s'%(str(e),str(args if args else kwargs)))
                        if exception_count['TypeError'] >= 1:
                            log.error('%s %s TypeError'%(str(e),str(args if args else kwargs)))
                            return
                        time.sleep(1)
                    except Exception as e:
                        exception_count['Exception'] += 1
                        log.exception(e)
                        time.sleep(5)
                else:
                    log.error('request fail %s'%str((args if args else kwargs)))
            return wrapper
        return decorate

headers= {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.8',
        'Connection': 'keep-alive',
        'Accept-Encoding': 'gzip, deflate',
        'cookie':'B=2h29jidbquh6b&b=3&s=sh; PRF=t%3DCVEO%252BEMLP%252BKONA'
        } 

class Yahoo():

    def __init__(self):
        self.session = requests.session()
        self.session.headers.update(headers)
        # self.session.get('https://xueqiu.com',timeout=10)

    @setLog.request()
    def get_us_profile(self, symbol):
        '''
        https://xueqiu.com/S/DMTX/keyratios
        '''

        url = 'https://finance.yahoo.com/quote/{symbol}/profile'.format_map({'symbol':symbol.replace('+','%2B')})
        req = self.session.get(url,timeout=10) 
        assert req.status_code == 200 ,requests.exceptions.RequestException

        if url not in req.url:
            print('No symbol %s'%symbol)
            return None

        m = re.search('^root.App.main = (?P<js>.*);$',req.text,re.M)
        dic = None
        if m:
            dic = json.loads(m.group('js'))
            dic = dic['context']['dispatcher']['stores']['QuoteSummaryStore']
            # print(dic)


        return dic


def format_html(html):
    m = re.search('^root.App.main = (?P<js>.*);$',html,re.M)
    # m = re.match('root.App.main = (.*?)',html)
    if m:
        dic = json.loads(m.group('js'))
        dic = dic['context']['dispatcher']['stores']['QuoteSummaryStore']
        # print(dic)
        js = json.dumps(dic)
        print(js)

    # return ret_dic

def test():
    # url = 'https://www.linkedin.com/company-beta/2114/jobs/'
    # url = 'https://finance.yahoo.com/quote/KONA/profile'
    # session = requests.Session()
    # session.headers.update(headers)
    # req = session.get(url)

    # with open('test/yahu.html','wb')as f:
    #     f.write(req.content)
    with open('test/yahu.html','rt')as f:
        html = f.read()
    format_html(html)

    # print(html)

def us():
    import pymongo
    Client = pymongo.MongoClient('192.168.1.220:29001')

    version = 1

    DB = Client.us_stock
    yh = Yahoo()

    symbols = [symbol.get('symbol') for symbol in DB['index'].find({},{'symbol':1})]
    finish = [symbol.get('symbol') for symbol in DB['yahoo_us_profile'].find({'version':version},{'symbol':1})]
    hi_symbols = set(symbols) - set(finish)

    td = tqdm(hi_symbols)
    for symbol in td:
        td.set_description('get_us_profile: %s'%symbol)
        dic = yh.get_us_profile(symbol)
        if dic:
            dic['_id'] = symbol
            dic['version'] = version
            DB['yahoo_us_profile'].insert_one(dic)

        # break

def hk():
    import pymongo
    Client = pymongo.MongoClient('192.168.1.220:29001')

    version = 1

    DB = Client.hk_stock
    yh = Yahoo()

    symbols = [symbol.get('symbol') for symbol in DB['index'].find({},{'symbol':1})]
    finish = [symbol.get('symbol') for symbol in DB['yahoo_hk_profile'].find({'version':version},{'symbol':1})]
    hi_symbols = set(symbols) - set(finish)

    td = tqdm(hi_symbols)
    for symbol in td:
        td.set_description('get_hk_profile: %s'%symbol)
        dic = yh.get_us_profile('%s.HK'%symbol[1::])
        if dic:
            dic['_id'] = symbol
            dic['version'] = version
            DB['yahoo_hk_profile'].insert_one(dic)

        # break


if __name__ == '__main__':
    hk()
    # test()